import com.hankcs.hanlp.HanLP
import com.hankcs.hanlp.dictionary.stopword.CoreStopWordDictionary
import com.hankcs.hanlp.summary.TextRankKeyword
import com.hankcs.hanlp.tokenizer.StandardTokenizer
import org.apache.spark.sql.{SparkSession, functions}

import scala.collection.convert.ImplicitConversions.`collection AsScalaIterable`
import scala.jdk.CollectionConverters.CollectionHasAsScala

object homework1 {



  def main(args: Array[String]): Unit = {
    val spark = SparkSession
      .builder
      .master("local[*]")
      .appName("Spark Pi")
      .getOrCreate()
    val sc = spark.sparkContext
    // 英文词频统计
    sc.textFile("F:\\Spark\\karry\\karry\\src\\main\\resources\\word.txt")
      .flatMap(_.split(" "))
      .map(x => (x, 1))
      .reduceByKey((x,y) => x + y)
    // 中文分词
    val chinese1 = HanLP.segment("张三上午没吃早餐，中午想吃牛肉面")
    // println(chinese1)
    //println(chinese1.asScala.map(_.word.trim))
    // 标准分词
    val chinese2 = StandardTokenizer.segment("明天放假++清明节++过三周还有假期++五一劳动节")
    println(chinese2)
    println(chinese2.asScala.map(_.word.replaceAll("\\s+", "")))
    //关键词提取
    TextRankKeyword.getKeywordList("速看！广东这五个地方将要拆迁，征收位置、面积、补偿和安置标准都已经公布，快来看看有你家吗？", 5)
      .forEach(println)
    //关键词过滤
    val chinese3 = """16:00:00 pm [spark课程] www.baidu.com""".split("\\s+")
    println(chinese3(2).replaceAll("\\[|\\]", ""))

    val textArr = Array(
    "su7爆燃遇难者母亲清空相关微博",
    "甲亢哥被重庆甜妹教育了",
    "胡歌老婆是他前助理",
    "金价再起飞",
    "你能接受与伴侣的年龄差是多少"
    )
    val textRDD = sc.parallelize(textArr)
    val textResult = textRDD.map {
      text =>
      val keyword = TextRankKeyword.getKeywordList(text,5).toString
      val words = transform(text)
        (text ,keyword, words)
    } // RDD[(String, String, List[String])]
    textResult.take(1).foreach(println)
    }
  def transform(sentense:String):List[String] ={
    val list = StandardTokenizer.segment(sentense)
    CoreStopWordDictionary.apply(list)
    list.map(x => x.word.replaceAll(" ","")).toList

    //所有电影中平均得分最高的前 10 部电影:
    // "ratings.dat"：UserID::MovieID::Rating::Timestamp

  }


  }
